{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/haithamelmarakeby/PycharmProjects/pnet2'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "from os.path import dirname, realpath, join\n",
    "base_dir = dirname(dirname(os.getcwd()))\n",
    "base_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, base_dir)\n",
    "from config_path import PROSTATE_DATA_PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from os.path import join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = join(PROSTATE_DATA_PATH, 'external_validation/Met500/samples.xlsx')\n",
    "met_df = pd.read_excel(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CASE</th>\n",
       "      <th>ID</th>\n",
       "      <th>AGE</th>\n",
       "      <th>GENDER</th>\n",
       "      <th>POPULATION</th>\n",
       "      <th>CANCER TYPE</th>\n",
       "      <th>HISTOLOG. TYPE</th>\n",
       "      <th>BIOPSY SITE</th>\n",
       "      <th>ANALYSIS COHORT</th>\n",
       "      <th>PRIMARY SITE</th>\n",
       "      <th>INTERVAL (mo)</th>\n",
       "      <th>PRIOR THERAPY</th>\n",
       "      <th>OTHER CANCER HISTORY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>MO_1002</td>\n",
       "      <td>49</td>\n",
       "      <td>F</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Cutaneous Melanoma</td>\n",
       "      <td>SKCM</td>\n",
       "      <td>Skin</td>\n",
       "      <td>SKCM</td>\n",
       "      <td>Skin</td>\n",
       "      <td>26</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>MO_1005</td>\n",
       "      <td>44</td>\n",
       "      <td>F</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Solitary Fibrous Tumor</td>\n",
       "      <td>SFT</td>\n",
       "      <td>Liver</td>\n",
       "      <td>SARC</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>108</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>MO_1007</td>\n",
       "      <td>62</td>\n",
       "      <td>F</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Extrahepatic Cholangiocarcinoma</td>\n",
       "      <td>EHCH</td>\n",
       "      <td>Liver</td>\n",
       "      <td>CHOL</td>\n",
       "      <td>Gall Bladder</td>\n",
       "      <td>6</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>MO_1008</td>\n",
       "      <td>47</td>\n",
       "      <td>M</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Prostate Adenocarcinoma</td>\n",
       "      <td>PRAD</td>\n",
       "      <td>Dura</td>\n",
       "      <td>PRAD</td>\n",
       "      <td>Prostate</td>\n",
       "      <td>41</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>MO_1011</td>\n",
       "      <td>69</td>\n",
       "      <td>M</td>\n",
       "      <td>Caucasian</td>\n",
       "      <td>Dedifferentiated Liposarcoma</td>\n",
       "      <td>DDLS</td>\n",
       "      <td>Shoulder</td>\n",
       "      <td>SARC</td>\n",
       "      <td>Retroperitoneum</td>\n",
       "      <td>124</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CASE       ID  AGE GENDER POPULATION                      CANCER TYPE  \\\n",
       "0     2  MO_1002   49      F  Caucasian               Cutaneous Melanoma   \n",
       "1     3  MO_1005   44      F  Caucasian           Solitary Fibrous Tumor   \n",
       "2     4  MO_1007   62      F  Caucasian  Extrahepatic Cholangiocarcinoma   \n",
       "3     5  MO_1008   47      M  Caucasian          Prostate Adenocarcinoma   \n",
       "4     6  MO_1011   69      M  Caucasian     Dedifferentiated Liposarcoma   \n",
       "\n",
       "  HISTOLOG. TYPE BIOPSY SITE ANALYSIS COHORT     PRIMARY SITE INTERVAL (mo)  \\\n",
       "0           SKCM        Skin            SKCM             Skin            26   \n",
       "1            SFT       Liver            SARC          Unknown           108   \n",
       "2           EHCH       Liver            CHOL     Gall Bladder             6   \n",
       "3           PRAD        Dura            PRAD         Prostate            41   \n",
       "4           DDLS    Shoulder            SARC  Retroperitoneum           124   \n",
       "\n",
       "  PRIOR THERAPY OTHER CANCER HISTORY  \n",
       "0             Y                  NaN  \n",
       "1             Y                  NaN  \n",
       "2             Y                  NaN  \n",
       "3             Y                  NaN  \n",
       "4             Y                  NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind= (met_df['CANCER TYPE'].str.lower().str.contains('prostate')) | (met_df['OTHER CANCER HISTORY'].str.lower().str.contains('prostate'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "met500_samples = met_df[ind]['ID'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(met500_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = join(PROSTATE_DATA_PATH,'processed/response_paper.csv')\n",
    "p1000_df = pd.read_csv(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AAPC-STID0000011640-Tumor-SM-2XU1H</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AAPC-STID0000021561-Tumor-SM-3RVWB</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AAPC-STID0000011949-Tumor-SM-2XU1I</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AAPC-STID0000021610-Tumor-SM-2XU13</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AAPC-STID0000021537-Tumor-SM-3RVW7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   id  response\n",
       "0  AAPC-STID0000011640-Tumor-SM-2XU1H         0\n",
       "1  AAPC-STID0000021561-Tumor-SM-3RVWB         0\n",
       "2  AAPC-STID0000011949-Tumor-SM-2XU1I         0\n",
       "3  AAPC-STID0000021610-Tumor-SM-2XU13         0\n",
       "4  AAPC-STID0000021537-Tumor-SM-3RVW7         0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1000_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "p1000_samples = p1000_df.id.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_samples = set(p1000_samples).intersection(set(met500_samples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "met500_samples = set(met500_samples).difference(common_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(met500_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'MO_1066',\n",
       " u'MO_1093',\n",
       " u'MO_1112',\n",
       " u'MO_1200',\n",
       " u'MO_1201',\n",
       " u'MO_1214',\n",
       " u'MO_1221',\n",
       " u'MO_1240',\n",
       " u'MO_1260',\n",
       " u'MO_1263',\n",
       " u'MO_1307',\n",
       " u'MO_1336',\n",
       " u'MO_1420',\n",
       " u'MO_1437',\n",
       " u'MO_1443',\n",
       " u'MO_1446',\n",
       " u'MO_1469',\n",
       " u'MO_1472',\n",
       " u'MO_1482',\n",
       " u'MO_1490',\n",
       " u'MO_1492',\n",
       " u'MO_1496',\n",
       " u'MO_1499',\n",
       " u'MO_1510',\n",
       " u'MO_1511',\n",
       " u'MO_1514',\n",
       " u'MO_1517',\n",
       " u'MO_1541',\n",
       " u'MO_1543',\n",
       " u'MO_1553',\n",
       " u'MO_1556',\n",
       " u'TP_2009',\n",
       " u'TP_2037',\n",
       " u'TP_2043',\n",
       " u'TP_2080',\n",
       " u'TP_2081',\n",
       " u'TP_2090',\n",
       " u'TP_2093',\n",
       " u'TP_2096',\n",
       " u'TP_2156'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "met500_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'MO_1008',\n",
       " u'MO_1012',\n",
       " u'MO_1013',\n",
       " u'MO_1014',\n",
       " u'MO_1015',\n",
       " u'MO_1020',\n",
       " u'MO_1040',\n",
       " u'MO_1074',\n",
       " u'MO_1084',\n",
       " u'MO_1094',\n",
       " u'MO_1095',\n",
       " u'MO_1096',\n",
       " u'MO_1114',\n",
       " u'MO_1118',\n",
       " u'MO_1124',\n",
       " u'MO_1128',\n",
       " u'MO_1130',\n",
       " u'MO_1132',\n",
       " u'MO_1139',\n",
       " u'MO_1161',\n",
       " u'MO_1162',\n",
       " u'MO_1176',\n",
       " u'MO_1179',\n",
       " u'MO_1184',\n",
       " u'MO_1192',\n",
       " u'MO_1202',\n",
       " u'MO_1215',\n",
       " u'MO_1219',\n",
       " u'MO_1232',\n",
       " u'MO_1241',\n",
       " u'MO_1244',\n",
       " u'MO_1249',\n",
       " u'MO_1262',\n",
       " u'MO_1277',\n",
       " u'MO_1316',\n",
       " u'MO_1337',\n",
       " u'MO_1339',\n",
       " u'MO_1410',\n",
       " u'MO_1421',\n",
       " u'MO_1447',\n",
       " u'MO_1460',\n",
       " u'MO_1473',\n",
       " u'TP_2001',\n",
       " u'TP_2010',\n",
       " u'TP_2020',\n",
       " u'TP_2032',\n",
       " u'TP_2034',\n",
       " u'TP_2054',\n",
       " u'TP_2060',\n",
       " u'TP_2061',\n",
       " u'TP_2064',\n",
       " u'TP_2069',\n",
       " u'TP_2077',\n",
       " u'TP_2078',\n",
       " u'TP_2079'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
